import random
import re
import time

from selenium import webdriver
from concurrent.futures import ThreadPoolExecutor,as_completed
from scrapy.selector import Selector
from multiprocessing import Queue
from selenium.webdriver.firefox.options import Options


import pymongo

import tushare as ts

#Firefox 无界面没设置
firefox_opt = Options()
firefox_opt.add_argument('--headless')
firefox_opt.add_argument('--disable-gpu')

#Firefox 设置代理


def get_mongo_col(col):
    myclient = pymongo.MongoClient("mongodb://192.168.0.21:27017")
    mydb = myclient['gupiao']
    mycol = mydb[col]
    return mycol


urls_queue = Queue()
def get_stock_urls():
    codes = ts.get_hs300s()['code'].tolist()
    for code in codes:
        urls_queue.put("http://finance.sina.com.cn/realstock/company/sh{0}/nc.shtml".format(code))


def get_stock_data(url):
    browser = webdriver.Firefox(firefox_options=firefox_opt)
    browser.get(url)
    time.sleep(random.randint(5,8))

    soup = Selector(text=browser.page_source)
    print(url)
    hq=soup.xpath('//div[@id="hq"]')
    table ={}
    name = str(soup.xpath('//*[@id="stockName"]/i/text()').extract_first()).strip()
    code = str(soup.xpath('//*[@id="stockName"]/span/text()').extract_first()).strip().replace(".SH","")
    price  = str(hq.xpath('.//*[@id="price"]/text()').extract_first()).strip()
    hqTime = str(hq.xpath('.//*[@id="hqTime"]/text()').extract_first()).strip()

    trs = []
    for i in range(1,6):
        tr={}
        tr_price=str(hq.xpath('.//*[@id="hqDetails"]/table/tbody/tr[{}]/td[1]/text()'.format(i)).extract_first()).strip()
        number=str(hq.xpath('.//*[@id="hqDetails"]/table/tbody/tr[{}]/td[2]/text()'.format(i)).extract_first()).strip()
        rate=str(hq.xpath('.//*[@id="hqDetails"]/table/tbody/tr[{}]/td[3]/text()'.format(i)).extract_first()).strip()
        tr['tr_price'+str(i)]= tr_price
        tr['number'+str(i)]=number
        tr['rate'+str(i)]=rate
        trs.append(tr)

    table['name']=name
    table['code'] = code
    table['price']=price
    table['hqTime']=hqTime
    table['tr'] =trs

    print(table)
    if price != 'None':
        colname = time.strftime("%Y_%m_%d", time.localtime())
        col=get_mongo_col(colname)
        col.insert(table)
    else:
        with open("error_code.txt",'a') as f:
            code = re.search('sh(\d+)',str(url))
            f.write(code.group(1)+"\n")

    browser.close()



get_stock_urls()
pool =ThreadPoolExecutor(max_workers=5)

while urls_queue.qsize() > 0 :
    pool.submit(get_stock_data,urls_queue.get())






